{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import transforms, datasets\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch.nn as nn\n",
    "import cv2\n",
    "from facenet_pytorch import MTCNN, InceptionResnetV1\n",
    "from PIL import Image\n",
    "\n",
    "# The model is running on CPU, since it is already pre-trained and doesnt require GPU\n",
    "device = torch.device('cpu') \n",
    "print('Running on device: {}'.format(device))\n",
    "\n",
    "#Define MTCNN module\n",
    "#Since MTCNN is a collection of neural nets and other code, \n",
    "#The device must be passed in the following way to enable copying of objects when needed internally.\n",
    "mtcnn = MTCNN(\n",
    "    image_size=160, margin=0, min_face_size=20,\n",
    "    thresholds=[0.6, 0.7, 0.7], factor=0.709, prewhiten=True,\n",
    "    device=device\n",
    ")\n",
    "#Function takes 2 vectors 'a' and 'b'\n",
    "#Returns the cosine similarity according to the definition of the dot product\n",
    "def cos_sim(a, b):\n",
    "    dot_product = np.dot(a, b)\n",
    "    norm_a = np.linalg.norm(a)\n",
    "    norm_b = np.linalg.norm(b)\n",
    "    return dot_product / (norm_a * norm_b)\n",
    "\n",
    "#cos_sim returns real numbers,where negative numbers have different interpretations.\n",
    "#So we use this function to return only positive values.\n",
    "def cos(a,b):\n",
    "    minx = -1 \n",
    "    maxx = 1\n",
    "    return (cos_sim(a,b)- minx)/(maxx-minx)\n",
    "\n",
    "# Define Inception Resnet V1 module (GoogLe Net)\n",
    "resnet = InceptionResnetV1(pretrained='vggface2').eval().to(device)\n",
    "\n",
    "# Define a dataset and data loader\n",
    "dataset = datasets.ImageFolder('Personal/Test')\n",
    "dataset.idx_to_class = {i:c for c, i in dataset.class_to_idx.items()}\n",
    "loader = DataLoader(dataset, collate_fn=lambda x: x[0])\n",
    "\n",
    "#Perfom MTCNN facial detection\n",
    "#Detects the face present in the image and prints the probablity of face detected in the image.\n",
    "aligned = []\n",
    "names = []\n",
    "for x, y in loader:\n",
    "    x_aligned, prob = mtcnn(x, return_prob=True)\n",
    "    if x_aligned is not None:\n",
    "        print('Face detected with probability: {:8f}'.format(prob))\n",
    "        aligned.append(x_aligned)\n",
    "        names.append(dataset.idx_to_class[y])\n",
    "\n",
    "# Calculate the 512 face embeddings\n",
    "aligned = torch.stack(aligned).to(device)\n",
    "embeddings = resnet(aligned).cpu()\n",
    "\n",
    "# Print distance matrix for classes.\n",
    "#The embeddings are plotted in space and cosine distace is measured.\n",
    "cos_sim = nn.CosineSimilarity(dim=-1, eps=1e-6)\n",
    "for i in range(0,len(names)):\n",
    "    emb=embeddings[i].unsqueeze(0)\n",
    "    # The cosine similarity between the embeddings is given by 'dist'.\n",
    "    dist =cos(embeddings[0],emb)  \n",
    "        \n",
    "dists = [[cos(e1,e2).item() for e2 in embeddings] for e1 in embeddings]\n",
    "# The print statement below is\n",
    "#Helpful for analysing the results and for determining the value of threshold.\n",
    "print(pd.DataFrame(dists, columns=names, index=names)) \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Face Recognition from Webcam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "from facenet_pytorch import MTCNN, InceptionResnetV1,extract_face\n",
    "from PIL import Image,ImageDraw\n",
    "import torch\n",
    "import cv2\n",
    "import os\n",
    "import torch.nn as nn\n",
    "from Web.imutils.video import WebcamVideoStream\n",
    "from Web.imutils.video import FPS\n",
    "import imutils\n",
    "\n",
    "\n",
    "i = 1\n",
    "# Haarcascade Classifier is used to detect faces through webcam. \n",
    "#It is preffered over MTCNN as it is faster. Real time basic applications needs to be fast.\n",
    "classifier = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')\n",
    "\n",
    "#Takes 2 vectors 'a' and 'b' .\n",
    "#Returns the cosine similarity according to the definition of the dot product.\n",
    "def cos_sim(a, b):\n",
    "    dot_product = np.dot(a, b)\n",
    "    norm_a = np.linalg.norm(a)\n",
    "    norm_b = np.linalg.norm(b)\n",
    "    return dot_product / (norm_a * norm_b)\n",
    "\n",
    "#cos_sim returns real numbers,where negative numbers have different interpretations.\n",
    "#So we use this function to return only positive values.\n",
    "def cos(a,b):\n",
    "    minx = -1 \n",
    "    maxx = 1\n",
    "    return (cos_sim(a,b)- minx)/(maxx-minx)\n",
    "\n",
    "#This is the function for doing face recognition.\n",
    "def verify(embedding):\n",
    "    for i,k in enumerate(embeddings):\n",
    "        for j,l in enumerate(embedding):\n",
    "            #Computing Cosine distance.\n",
    "            dist =cos(k,l)\n",
    "            \n",
    "            #Chosen threshold is 0.85\n",
    "            #Threshold is determined after seeing the table in the previous cell.\n",
    "            if dist > 0.85:\n",
    "                #Name of the person identified is printed on the screen, as well as below the detecetd face (below the rectangular box).\n",
    "                text= names[i]\n",
    "                cv2.putText(ima, text,(boxes[j][0].astype(int),boxes[j][3].astype(int) + 17), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (255,255,255), 2)\n",
    "                print(text)\n",
    "        \n",
    "#Model running on CPU           \n",
    "device = torch.device('cpu')  \n",
    "\n",
    "#Define Inception Resnet V1 module (GoogLe Net)                    \n",
    "resnet = InceptionResnetV1(pretrained='vggface2').eval().to('cpu')\n",
    "\n",
    "#Define MTCNN module\n",
    "#Since MTCNN is a collection of neural nets and other code, \n",
    "#The device must be passed in the following way to enable copying of objects when needed internally.\n",
    "#'keep_all' is kept True. All the faces present in the image will be detected.\n",
    "mtcnn = MTCNN(\n",
    "    image_size=160, margin=0, min_face_size=20,\n",
    "    thresholds=[0.6, 0.7, 0.7], factor=0.709, prewhiten=True,\n",
    "    device=device,keep_all=True\n",
    ")\n",
    "\n",
    "#Camera is opened. Webcam video streaming starts.\n",
    "vs = WebcamVideoStream(src=0).start()\n",
    "print(\"Camera on\")\n",
    "while True:\n",
    "    im= vs.read()\n",
    "    #Flip to act as a mirror\n",
    "    im=cv2.flip(im,1) \n",
    "    \n",
    "    try:\n",
    "        #The resize function of imutils maintains the aspect ratio \n",
    "        #It provides the keyword arguments width and heightso the image can be resized to the intended width/height \n",
    "        frame = imutils.resize(im, width=400)\n",
    "        \n",
    "        #Detecting faces using Haarcascade classifier.\n",
    "        faces = classifier.detectMultiScale(frame)\n",
    "        path=\"./Personal/Pics/\".format(i)\n",
    "        img_name = \"image_{}.jpg\".format(i)  \n",
    "        #The captured image is saved.\n",
    "        cv2.imwrite(os.path.join(path,img_name),frame)\n",
    "        imgName=\"./Personal/Pics/image_{}.jpg\".format(i)\n",
    "        \n",
    "        # Get cropped and prewhitened image tensor\n",
    "        img=Image.open(imgName)\n",
    "        i=i+1\n",
    "        img_cropped = mtcnn(img)\n",
    "        boxes,prob=mtcnn.detect(img)\n",
    "        img_draw = img.copy()\n",
    "        draw = ImageDraw.Draw(img_draw)\n",
    "        \n",
    "        #Rectangular boxes are drawn on faces present in the image.\n",
    "        #The detected and cropped faces are then saved.\n",
    "        for i, box in enumerate(boxes):\n",
    "            draw.rectangle(box.tolist())\n",
    "            extract_face(img, box, save_path='./Personal/Pics/Cropped_Face_{}.jpg'.format(i))\n",
    "        img_draw.save('./Personal/Pics/Faces_Detected.jpg')\n",
    "        ima=cv2.imread('./Personal/Pics/Faces_Detected.jpg')\n",
    "        \n",
    "        #Calculate embeddings of each cropped face.\n",
    "        img_embedding = resnet(img_cropped)\n",
    "        \n",
    "        #Call function verify. \n",
    "        #Identify the person with the help of embeddings.\n",
    "        cos_sim = nn.CosineSimilarity(dim=-1, eps=1e-6)\n",
    "        verify(img_embedding)\n",
    "        \n",
    "        #'Detecting..' window opens.\n",
    "        #Rectangular boxes are drawn on detected faces.\n",
    "        #The identified faces have their respective name below the box.\n",
    "        cv2.imshow('Detecting...',ima)\n",
    "    \n",
    "    except:\n",
    "        #In case 'try' doesn't work, \"Get the image embedding\" text is printed on the screen.\n",
    "        #Run first cell\n",
    "        text=\"Get the image embeddings\"\n",
    "        print(text)\n",
    "        break \n",
    "        \n",
    "    key = cv2.waitKey(1)\n",
    "    \n",
    "    #13 is for 'Enter' key.\n",
    "    #If 'Enter' key is pressed, all the windows are made to close forcefully.\n",
    "    if key ==13:\n",
    "        break    \n",
    "        \n",
    "cv2.destroyAllWindows() \n",
    "vs.stop()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
